# -*- coding: utf-8 -*-
'''
Created on 01-12-2012

@author: LONG HOANG GIANG
'''
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib2 import commonlib, html2text
from Model.storymodelv3 import StoryModel
from urlparse import urljoin
import datetime
import re
import workerpool


class Alobooks():
    
    xpathlst = ["//div[@class='content node-forum']/div[1]", "//div[contains(@class, 'content node')]/div[1]"]
    
    def test(self):
        
        url = 'http://alobooks.vn/forum/15770/cho-anh-nhin-ve-em-tan-di-o-tap-2-full.html'
        html = commonlib.loadweb(url).gethtml()
        commonlib.file_put_contents("/test/0001.html", html)
    
    def getDetail(self, url, **kw):
        tree =  commonlib.loadweb(url).build_tree()
        title = kw.get('title')
        for xpathstr in self.xpathlst:
            contentNode = tree.xpath(xpathstr)
            if len(contentNode) < 1: continue
            html = commonlib.Etree.tostring(contentNode[0])
            html = commonlib.getHtmlContent(html)
            html = html2text.html2text(html, w100=True).strip()
#            html = commonlib.toUnicodeDungSan(html)
            html = re.sub(ur"( *\n *){3,}", "\n\n", html)
            html = re.sub(ur"\n", "<br />", html)
            html = html.replace("previous page next page", '')
            if re.search(ur"(Chương[^<]+)", html):
                html = re.sub(ur"(Chương[^<]+)", "<b>\\1</b><br /><hr /><br />", html)
            else:
                html = "<b>{0}</b><br /><hr /><br />{1}".format(title, html)
            html = html.replace("<br /><br /> <br /><br />", "<br />")
            return html
        return None
    
    def normalizeTitle(self, title):
#        title = title
#        if '-' in title:
#            title = title[title.find('-') + 1:].strip()
#            if title == '': title = title
#        
#        return title
        return title
    
    def processOneStory(self, sUrl):
        tree = commonlib.loadweb(sUrl).build_tree()
        if tree == None: return
        data = [{"title": "GIỚI THIỆU", "url": sUrl}]
        data = []
        
        for item in tree.xpath("//div[contains(@id, 'book-navigation')]/ul/li/a"):
            href = commonlib.normalize_str(item.get('href', ''))
            if href == '': continue
            url = urljoin(sUrl, href)
            name = commonlib.toUpper(self.normalizeTitle(commonlib.stringify(item)))
            data.append({'title': name, 'url': url})
        
        def load(item):
            commonlib.loadweb(item['url'])
            
        pool = workerpool.WorkerPool(size=5)
        pool.map(load, data)
        pool.shutdown()
        pool.wait()
        
        db = StoryModel(DATABASE)
        db.open(True)
        for item in data:
            detail = self.getDetail(item['url'], title=item['title'])
            db.add_story(item['title'], detail, 0)
            print '______________________'
            print item['title']
            print '______________________'
            print detail, "\n\n"
        db.close()
            
    def getStory(self, item):
        url = item['url']
        tree = commonlib.loadweb(url).build_tree()
        if tree == None: return
        data = [{"title": "GIỚI THIỆU", "url": url}]
        for item in tree.xpath("//div[contains(@id, 'book-navigation')]/ul/li/a"):
            href = commonlib.normalize_str(item.get('href', ''))
            if href == '': continue
            clink = urljoin(url, href)
            name = commonlib.toUpper(self.normalizeTitle(commonlib.stringify(item)))
            data.append({'title': name, 'url': clink})
        
        def load(item):
            commonlib.loadweb(item['url'])
            
        pool = workerpool.WorkerPool(size=5)
        pool.map(load, data)
        pool.shutdown()
        pool.wait()
        
        detailOfStory = []
        for item in data:
            detail = self.getDetail(item['url'])
            detailOfStory.append({'title': item['title'], 'detail': detail})
        return detailOfStory
    
    def getMultipleStory(self, url='http://alobooks.vn/forum/4/ngon-tinh-trung-quoc.aspx'):
        catcachefile = '/longhoanggiang/cache/alobook_cat_cache'
        cat = commonlib.pickle_load(catcachefile)
        if cat == None:
            cat = []
            c = 0
            while c < 1000: 
                tree = commonlib.loadweb(url).build_tree()
                for node in tree.xpath("//div[@id='forum']/table/tbody/tr/td[@class='title']/div/a"):
                    name = commonlib.toUpper(commonlib.stringify(node))
                    link = node.get('href', '').strip()
                    if link == '' or name == '': continue
                    link = urljoin(url, link)
                    print name, link
                    id = commonlib.crc32unsigned(link)
                    cat.append({'id': id, 'name': name, 'url': link})
                
                nextPageNode = tree.xpath("//ul[@class='pager']/li[contains(@class, 'pager-current')]/following-sibling::li[1]/a")
                if len(nextPageNode) == 0: break
                link = nextPageNode[0].get('href', '').strip()
                if link == '': break
                url = urljoin(url, link)
                c += 1
            commonlib.pickle_dump(cat, catcachefile)
        db = StoryModel(DATABASE)
        db.open(True)
        for c in cat:
            print c['name']
            cid = db.add_category(c['name'])
            data = self.getStory(c)
            for item in data:
                detail = "<b>{0}</b><br /><hr /><br />{1}".format(item['title'], item['detail'])
                db.add_story(item['title'], detail, cid)
                print '_________________________________'
                print item['title'], cid
                print '_________________________________'
                print detail, "\n"
        db.close()

if __name__ == '__main__':
    
    DATABASE = '/longhoanggiang/database/{0}'.format('noilaanhoedi')
    
    alobook = Alobooks()
    alobook.processOneStory('http://alobooks.vn/doc-sach-truc-tuyen/25757/noi-la-anh-nho-em-di-full-phan-anh.html')
#    alobook.getMultipleStory()
    
    print '> Finished at {0}'.format(datetime.datetime.now())
    os._exit(1)